<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Intro to Policy Optimization 代码详解 | 云玩家</title><meta name="keywords" content="机器学习,Python,强化学习,代码实例,Pytorch"><meta name="author" content="云玩家"><meta name="copyright" content="云玩家"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="本篇文章是 OpenAI Spinnging Up 中 Part 3: Intro to Policy Optimization 中代码的学习笔记, 原文在 https:&#x2F;&#x2F;spinningup.openai.com&#x2F;en&#x2F;latest&#x2F;spinningup&#x2F;rl_intro3.html , 代码在 https:&#x2F;&#x2F;github.com&#x2F;openai&#x2F;spinningup&#x2F;blob&#x2F;master&#x2F;">
<meta property="og:type" content="article">
<meta property="og:title" content="Intro to Policy Optimization 代码详解">
<meta property="og:url" content="http://yunist.cn/ML/RL/primer/Intro_to_Policy_Optimization_code/index.html">
<meta property="og:site_name" content="云玩家">
<meta property="og:description" content="本篇文章是 OpenAI Spinnging Up 中 Part 3: Intro to Policy Optimization 中代码的学习笔记, 原文在 https:&#x2F;&#x2F;spinningup.openai.com&#x2F;en&#x2F;latest&#x2F;spinningup&#x2F;rl_intro3.html , 代码在 https:&#x2F;&#x2F;github.com&#x2F;openai&#x2F;spinningup&#x2F;blob&#x2F;master&#x2F;">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="http://yunist.cn/ML/RL/primer/Intro_to_Policy_Optimization_code/Intro_to_Policy_Optimization_code.png">
<meta property="article:published_time" content="2020-06-25T02:34:24.000Z">
<meta property="article:modified_time" content="2021-08-14T04:39:34.294Z">
<meta property="article:author" content="云玩家">
<meta property="article:tag" content="机器学习">
<meta property="article:tag" content="Python">
<meta property="article:tag" content="强化学习">
<meta property="article:tag" content="代码实例">
<meta property="article:tag" content="Pytorch">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="http://yunist.cn/ML/RL/primer/Intro_to_Policy_Optimization_code/Intro_to_Policy_Optimization_code.png"><link rel="shortcut icon" href="/img/favicon.ico"><link rel="canonical" href="http://yunist.cn/ML/RL/primer/Intro_to_Policy_Optimization_code/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//s4.cnzz.com"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><script async="async" data-pjax="data-pjax" src="https://s4.cnzz.com/z_stat.php?id=1278869595&amp;web_id=1278869595"></script><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: undefined,
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    jQuery: 'https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js',
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
    },
    fancybox: {
      js: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js',
      css: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isanchor: false
}</script><script src="/js/yuntools.js"></script><script id="config-diff">var GLOBAL_CONFIG_SITE = { 
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2021-08-14 12:39:34'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    })(window)</script><link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/cnyist/blog/css/font.css"><meta name="generator" content="Hexo 5.4.0"></head><body><div id="web_bg"></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="author-avatar"><img class="avatar-img" src="https://cdn.jsdelivr.net/gh/cnyist/blog/img/avatar.svg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/archives/"><div class="headline">文章</div><div class="length-num">97</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/tags/"><div class="headline">标签</div><div class="length-num">40</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/categories/"><div class="headline">分类</div><div class="length-num">27</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/chat/"><i class="fa-fw fas fa-comments"></i><span> 闲聊室</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-user"></i><span> about</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw iconfont icon-Web"></i><span> 镜像站点</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" target="_blank" rel="noopener" href="https://yunist.gitee.io"><i class="fa-fw iconfont icon-gitee-fill-round"></i><span> Gitee Pages</span></a></li></ul></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('/ML/RL/primer/Intro_to_Policy_Optimization_code/Intro_to_Policy_Optimization_code.png')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">云玩家</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/chat/"><i class="fa-fw fas fa-comments"></i><span> 闲聊室</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-user"></i><span> about</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw iconfont icon-Web"></i><span> 镜像站点</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" target="_blank" rel="noopener" href="https://yunist.gitee.io"><i class="fa-fw iconfont icon-gitee-fill-round"></i><span> Gitee Pages</span></a></li></ul></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">Intro to Policy Optimization 代码详解</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2020-06-25T02:34:24.000Z" title="发表于 2020-06-25 10:34:24">2020-06-25</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2021-08-14T04:39:34.294Z" title="更新于 2021-08-14 12:39:34">2021-08-14</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/">机器学习</a><i class="fas fa-angle-right post-meta-separator"></i><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/%E5%BC%BA%E5%8C%96%E5%AD%A6%E4%B9%A0/">强化学习</a><i class="fas fa-angle-right post-meta-separator"></i><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/%E5%BC%BA%E5%8C%96%E5%AD%A6%E4%B9%A0/%E5%85%A5%E9%97%A8/">入门</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span><span class="post-meta-separator">|</span><span class="post-meta-commentcount"><i class="far fa-comments fa-fw post-meta-icon"></i><span class="post-meta-label">评论数:</span><a href="/ML/RL/primer/Intro_to_Policy_Optimization_code/#post-comment"><span id="twikoo-count"></span></a></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><p>本篇文章是 OpenAI Spinnging Up 中 Part 3: Intro to Policy Optimization 中代码的学习笔记, 原文在 <a target="_blank" rel="noopener" href="https://spinningup.openai.com/en/latest/spinningup/rl_intro3.html">https://spinningup.openai.com/en/latest/spinningup/rl_intro3.html</a> , 代码在 <a target="_blank" rel="noopener" href="https://github.com/openai/spinningup/blob/master/spinup/examples/pytorch/pg_math/1_simple_pg.py">https://github.com/openai/spinningup/blob/master/spinup/examples/pytorch/pg_math/1_simple_pg.py</a> .</p>
<p>先给出代码</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> torch.nn <span class="keyword">as</span> nn</span><br><span class="line"><span class="keyword">from</span> torch.distributions.categorical <span class="keyword">import</span> Categorical</span><br><span class="line"><span class="keyword">from</span> torch.optim <span class="keyword">import</span> Adam</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> gym</span><br><span class="line"><span class="keyword">from</span> gym.spaces <span class="keyword">import</span> Discrete, Box</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">mlp</span>(<span class="params">sizes, activation=nn.Tanh, output_activation=nn.Identity</span>):</span></span><br><span class="line">    <span class="comment"># Build a feedforward neural network.</span></span><br><span class="line">    layers = []</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="built_in">len</span>(sizes)-<span class="number">1</span>):</span><br><span class="line">        act = activation <span class="keyword">if</span> j &lt; <span class="built_in">len</span>(sizes)-<span class="number">2</span> <span class="keyword">else</span> output_activation</span><br><span class="line">        layers += [nn.Linear(sizes[j], sizes[j+<span class="number">1</span>]), act()]</span><br><span class="line">    <span class="keyword">return</span> nn.Sequential(*layers)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">train</span>(<span class="params">env_name=<span class="string">&#x27;CartPole-v0&#x27;</span>, hidden_sizes=[<span class="number">32</span>], lr=<span class="number">1e-2</span>, </span></span></span><br><span class="line"><span class="function"><span class="params">          epochs=<span class="number">50</span>, batch_size=<span class="number">5000</span>, render=<span class="literal">False</span></span>):</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># make environment, check spaces, get obs / act dims</span></span><br><span class="line">    env = gym.make(env_name)</span><br><span class="line">    <span class="keyword">assert</span> <span class="built_in">isinstance</span>(env.observation_space, Box), \</span><br><span class="line">        <span class="string">&quot;This example only works for envs with continuous state spaces.&quot;</span></span><br><span class="line">    <span class="keyword">assert</span> <span class="built_in">isinstance</span>(env.action_space, Discrete), \</span><br><span class="line">        <span class="string">&quot;This example only works for envs with discrete action spaces.&quot;</span></span><br><span class="line"></span><br><span class="line">    obs_dim = env.observation_space.shape[<span class="number">0</span>]</span><br><span class="line">    n_acts = env.action_space.n</span><br><span class="line"></span><br><span class="line">    <span class="comment"># make core of policy network</span></span><br><span class="line">    logits_net = mlp(sizes=[obs_dim]+hidden_sizes+[n_acts])</span><br><span class="line"></span><br><span class="line">    <span class="comment"># make function to compute action distribution</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get_policy</span>(<span class="params">obs</span>):</span></span><br><span class="line">        logits = logits_net(obs)</span><br><span class="line">        <span class="keyword">return</span> Categorical(logits=logits)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># make action selection function (outputs int actions, sampled from policy)</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get_action</span>(<span class="params">obs</span>):</span></span><br><span class="line">        <span class="keyword">return</span> get_policy(obs).sample().item()</span><br><span class="line"></span><br><span class="line">    <span class="comment"># make loss function whose gradient, for the right data, is policy gradient</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">compute_loss</span>(<span class="params">obs, act, weights</span>):</span></span><br><span class="line">        logp = get_policy(obs).log_prob(act)</span><br><span class="line">        <span class="keyword">return</span> -(logp * weights).mean()</span><br><span class="line"></span><br><span class="line">    <span class="comment"># make optimizer</span></span><br><span class="line">    optimizer = Adam(logits_net.parameters(), lr=lr)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># for training policy</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">train_one_epoch</span>():</span></span><br><span class="line">        <span class="comment"># make some empty lists for logging.</span></span><br><span class="line">        batch_obs = []          <span class="comment"># for observations</span></span><br><span class="line">        batch_acts = []         <span class="comment"># for actions</span></span><br><span class="line">        batch_weights = []      <span class="comment"># for R(tau) weighting in policy gradient</span></span><br><span class="line">        batch_rets = []         <span class="comment"># for measuring episode returns</span></span><br><span class="line">        batch_lens = []         <span class="comment"># for measuring episode lengths</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># reset episode-specific variables</span></span><br><span class="line">        obs = env.reset()       <span class="comment"># first obs comes from starting distribution</span></span><br><span class="line">        done = <span class="literal">False</span>            <span class="comment"># signal from environment that episode is over</span></span><br><span class="line">        ep_rews = []            <span class="comment"># list for rewards accrued throughout ep</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># render first episode of each epoch</span></span><br><span class="line">        finished_rendering_this_epoch = <span class="literal">False</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># collect experience by acting in the environment with current policy</span></span><br><span class="line">        <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line"></span><br><span class="line">            <span class="comment"># rendering</span></span><br><span class="line">            <span class="keyword">if</span> (<span class="keyword">not</span> finished_rendering_this_epoch) <span class="keyword">and</span> render:</span><br><span class="line">                env.render()</span><br><span class="line"></span><br><span class="line">            <span class="comment"># save obs</span></span><br><span class="line">            batch_obs.append(obs.copy())</span><br><span class="line"></span><br><span class="line">            <span class="comment"># act in the environment</span></span><br><span class="line">            act = get_action(torch.as_tensor(obs, dtype=torch.float32))</span><br><span class="line">            obs, rew, done, _ = env.step(act)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># save action, reward</span></span><br><span class="line">            batch_acts.append(act)</span><br><span class="line">            ep_rews.append(rew)</span><br><span class="line"></span><br><span class="line">            <span class="keyword">if</span> done:</span><br><span class="line">                <span class="comment"># if episode is over, record info about episode</span></span><br><span class="line">                ep_ret, ep_len = <span class="built_in">sum</span>(ep_rews), <span class="built_in">len</span>(ep_rews)</span><br><span class="line">                batch_rets.append(ep_ret)</span><br><span class="line">                batch_lens.append(ep_len)</span><br><span class="line"></span><br><span class="line">                <span class="comment"># the weight for each logprob(a|s) is R(tau)</span></span><br><span class="line">                batch_weights += [ep_ret] * ep_len</span><br><span class="line"></span><br><span class="line">                <span class="comment"># reset episode-specific variables</span></span><br><span class="line">                obs, done, ep_rews = env.reset(), <span class="literal">False</span>, []</span><br><span class="line"></span><br><span class="line">                <span class="comment"># won&#x27;t render again this epoch</span></span><br><span class="line">                finished_rendering_this_epoch = <span class="literal">True</span></span><br><span class="line"></span><br><span class="line">                <span class="comment"># end experience loop if we have enough of it</span></span><br><span class="line">                <span class="keyword">if</span> <span class="built_in">len</span>(batch_obs) &gt; batch_size:</span><br><span class="line">                    <span class="keyword">break</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># take a single policy gradient update step</span></span><br><span class="line">        optimizer.zero_grad()</span><br><span class="line">        batch_loss = compute_loss(obs=torch.as_tensor(batch_obs, dtype=torch.float32),</span><br><span class="line">                                  act=torch.as_tensor(batch_acts, dtype=torch.int32),</span><br><span class="line">                                  weights=torch.as_tensor(batch_weights, dtype=torch.float32)</span><br><span class="line">                                  )</span><br><span class="line">        batch_loss.backward()</span><br><span class="line">        optimizer.step()   </span><br><span class="line">        <span class="keyword">return</span> batch_loss, batch_rets, batch_lens</span><br><span class="line"></span><br><span class="line">    <span class="comment"># training loop</span></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(epochs):</span><br><span class="line">        batch_loss, batch_rets, batch_lens = train_one_epoch()</span><br><span class="line">        print(<span class="string">&#x27;epoch: %3d \t loss: %.3f \t return: %.3f \t ep_len: %.3f&#x27;</span>%</span><br><span class="line">                (i, batch_loss, np.mean(batch_rets), np.mean(batch_lens)))</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    <span class="keyword">import</span> argparse</span><br><span class="line">    parser = argparse.ArgumentParser()</span><br><span class="line">    parser.add_argument(<span class="string">&#x27;--env_name&#x27;</span>, <span class="string">&#x27;--env&#x27;</span>, <span class="built_in">type</span>=<span class="built_in">str</span>, default=<span class="string">&#x27;CartPole-v0&#x27;</span>)</span><br><span class="line">    parser.add_argument(<span class="string">&#x27;--render&#x27;</span>, action=<span class="string">&#x27;store_true&#x27;</span>)</span><br><span class="line">    parser.add_argument(<span class="string">&#x27;--lr&#x27;</span>, <span class="built_in">type</span>=<span class="built_in">float</span>, default=<span class="number">1e-2</span>)</span><br><span class="line">    args, unknown = parser.parse_known_args()</span><br><span class="line">    print(<span class="string">&#x27;\nUsing simplest formulation of policy gradient.\n&#x27;</span>)</span><br><span class="line">    train(env_name=args.env_name, render=args.render, lr=args.lr)</span><br></pre></td></tr></table></figure>

<p>这里我们会对大部分函数以及一些变量一一解析, 其中一些 Pytorch 的 API 可以参考我的<a href="https://yunist.cn/ML/framework/Pytroch/pytorch_api/">这篇文章</a>或者<a target="_blank" rel="noopener" href="https://pytorch.org/docs/stable/index.html">官方文档</a> .</p>
<h1 id="mlp"><a href="#mlp" class="headerlink" title="mlp"></a>mlp</h1><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">mlp</span>(<span class="params">sizes, activation=nn.Tanh, output_activation=nn.Identity</span>):</span></span><br><span class="line">    <span class="comment"># Build a feedforward neural network.</span></span><br><span class="line">    layers = []</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="built_in">len</span>(sizes)-<span class="number">1</span>):</span><br><span class="line">        act = activation <span class="keyword">if</span> j &lt; <span class="built_in">len</span>(sizes)-<span class="number">2</span> <span class="keyword">else</span> output_activation</span><br><span class="line">        layers += [nn.Linear(sizes[j], sizes[j+<span class="number">1</span>]), act()]</span><br><span class="line">    <span class="keyword">return</span> nn.Sequential(*layers)</span><br></pre></td></tr></table></figure>

<p>依据输入返回一个神经网络.</p>
<h2 id="参数"><a href="#参数" class="headerlink" title="参数"></a>参数</h2><ul>
<li><p><code>sizes</code></p>
<p>其中包含神经网络的层数以及节点数信息</p>
</li>
<li><p><code>activation</code></p>
<p>节点的激活函数, 这里默认是 <code>nn.Tanh</code> 也就是 $\tanh$ 函数</p>
</li>
<li><p><code>output_activation</code></p>
<p>输出的激活函数</p>
</li>
</ul>
<h2 id="解析"><a href="#解析" class="headerlink" title="解析"></a>解析</h2><p><code>layers</code> 中的每一个元素就是神经网络的一部分 (节点与激活函数), 而 <code>nn.Sequential(*layers)</code> 是将这些部分组合成一个神经网络. 其中</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="built_in">len</span>(sizes)-<span class="number">1</span>):</span><br><span class="line">        act = activation <span class="keyword">if</span> j &lt; <span class="built_in">len</span>(sizes)-<span class="number">2</span> <span class="keyword">else</span> output_activation</span><br><span class="line">        layers += [nn.Linear(sizes[j], sizes[j+<span class="number">1</span>]), act()]</span><br></pre></td></tr></table></figure>

<p>这个循环, <code>act</code> 指的是激活函数, 当该层不是最后一层时使用 <code>activation</code> , 是时使用 <code>output_activation</code> 作为激活函数.</p>
<h1 id="get-policy"><a href="#get-policy" class="headerlink" title="get_policy"></a>get_policy</h1><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_policy</span>(<span class="params">obs</span>):</span></span><br><span class="line">    logits = logits_net(obs)</span><br><span class="line">    <span class="keyword">return</span> Categorical(logits=logits)</span><br></pre></td></tr></table></figure>

<p>依据环境计算出动作的对数概率, 并依此返回一个 <code>Categorical</code> 对象.</p>
<h2 id="参数-1"><a href="#参数-1" class="headerlink" title="参数"></a>参数</h2><ul>
<li><p><code>obs</code></p>
<p>环境的参数, 描述了环境</p>
</li>
</ul>
<h2 id="解析-1"><a href="#解析-1" class="headerlink" title="解析"></a>解析</h2><p><code>logits_net</code> 是一个神经网络, 接受参数后输出最终结果 (动作的对数概率). 至于 <code>Categorical</code> 对象请自行了解.</p>
<h1 id="get-action"><a href="#get-action" class="headerlink" title="get_action"></a>get_action</h1><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_action</span>(<span class="params">obs</span>):</span></span><br><span class="line">    <span class="keyword">return</span> get_policy(obs).sample().item()</span><br></pre></td></tr></table></figure>

<h2 id="参数-2"><a href="#参数-2" class="headerlink" title="参数"></a>参数</h2><ul>
<li><p><code>obs</code></p>
<p>环境的参数, 描述了环境.</p>
</li>
</ul>
<h2 id="解析-2"><a href="#解析-2" class="headerlink" title="解析"></a>解析</h2><p>利用 <code>Categorical</code> 对象采样动作.</p>
<h1 id="compute-loss"><a href="#compute-loss" class="headerlink" title="compute_loss"></a>compute_loss</h1><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">compute_loss</span>(<span class="params">obs, act, weights</span>):</span></span><br><span class="line">    logp = get_policy(obs).log_prob(act)</span><br><span class="line">    <span class="keyword">return</span> -(logp * weights).mean()</span><br></pre></td></tr></table></figure>

<p>计算损失.</p>
<h2 id="参数-3"><a href="#参数-3" class="headerlink" title="参数"></a>参数</h2><ul>
<li><p><code>obs</code></p>
<p>环境的参数, 描述了环境</p>
</li>
<li><p><code>act</code></p>
<p>采样的动作</p>
</li>
<li><p><code>weights</code></p>
<p>某项的权重</p>
</li>
</ul>
<h2 id="解析-3"><a href="#解析-3" class="headerlink" title="解析"></a>解析</h2><p>损失函数对参数的梯度要和期望回报对参数的梯度相同, 而期望回报对参数的梯度的估计式为<br>$$<br>\hat{g}=\frac{1}{|\mathcal{D}|}\sum_{\tau\in\mathcal{D}}\sum^T_{t=0}\nabla_\theta\log \pi_\theta(a_t\mid s_t)R(\tau)<br>$$<br><code>logp</code> 其实就是 $\log \pi_\theta(a_t\mid s_t)$ , 而 <code>weight</code> 其实就是 $R(\tau)$ . 因此该函数返回的其实就是<br>$$<br>\frac{1}{|\mathcal{D}|}\sum_{\tau\in\mathcal{D}}\sum^T_{t=0}\log \pi_\theta(a_t\mid s_t)R(\tau)<br>$$<br>对 $\theta$ 求导后正是我们的梯度.</p>
<h1 id="train-one-epoch"><a href="#train-one-epoch" class="headerlink" title="train_one_epoch"></a>train_one_epoch</h1><p>这是训练一个 epoch 的函数 (神经网络参数更新一次) .</p>
<h2 id="解析-4"><a href="#解析-4" class="headerlink" title="解析"></a>解析</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">if</span> done:</span><br><span class="line">    <span class="comment"># if episode is over, record info about episode</span></span><br><span class="line">    ep_ret, ep_len = <span class="built_in">sum</span>(ep_rews), <span class="built_in">len</span>(ep_rews)</span><br><span class="line">    batch_rets.append(ep_ret)</span><br><span class="line">    batch_lens.append(ep_len)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># the weight for each logprob(a|s) is R(tau)</span></span><br><span class="line">    batch_weights += [ep_ret] * ep_len</span><br><span class="line"></span><br><span class="line">    <span class="comment"># reset episode-specific variables</span></span><br><span class="line">    obs, done, ep_rews = env.reset(), <span class="literal">False</span>, []</span><br><span class="line"></span><br><span class="line">    <span class="comment"># won&#x27;t render again this epoch</span></span><br><span class="line">    finished_rendering_this_epoch = <span class="literal">True</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># end experience loop if we have enough of it</span></span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">len</span>(batch_obs) &gt; batch_size:</span><br><span class="line">        <span class="keyword">break</span></span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># take a single policy gradient update step</span></span><br><span class="line">optimizer.zero_grad()</span><br><span class="line">batch_loss = compute_loss(obs=torch.as_tensor(batch_obs, dtype=torch.float32),</span><br><span class="line">                          act=torch.as_tensor(batch_acts, dtype=torch.int32),</span><br><span class="line">                          weights=torch.as_tensor(batch_weights, dtype=torch.float32)</span><br><span class="line">                          )</span><br><span class="line">batch_loss.backward()</span><br><span class="line">optimizer.step()   </span><br></pre></td></tr></table></figure>

<p>依据 <code>batch_size</code> 确定走一个 epoch 走多少步. 然后当某个轨迹结束时 (也就是 <code>done</code> ) , 会计算总的回报, 然后通过 <code>compute_loss</code> 计算损失, 同时通过 Pytorch 的自动求导机制算出梯度, 然后用 <code>optimizer</code> (Adam 算法) 更新.</p>
<h1 id="train"><a href="#train" class="headerlink" title="train"></a>train</h1><p>整个过程其实就是重复多个 epoch , 然后最终训练 <code>epochs</code> 次. 如果需要利用 Gym 的可视化, 可以将 <code>render</code> 参数设为 <code>True</code></p>
<h2 id="参数-4"><a href="#参数-4" class="headerlink" title="参数"></a>参数</h2><ul>
<li><p><code>env_name</code></p>
<p>Gym 环境的名称</p>
</li>
<li><p><code>hidden_sizes</code></p>
<p>神经网络隐藏节点数, 可以自行调整</p>
</li>
<li><p><code>lr</code></p>
<p>学习率.</p>
</li>
<li><p><code>epochs</code></p>
<p>训练的 epoch 数</p>
</li>
<li><p><code>batch_size</code></p>
<p>一次 epoch 行动的次数</p>
</li>
<li><p><code>render</code></p>
<p>Gym 是否可视化 (<code>True</code> or <code>False</code>)</p>
</li>
</ul>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined">云玩家</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="http://yunist.cn/ML/RL/primer/Intro_to_Policy_Optimization_code/">http://yunist.cn/ML/RL/primer/Intro_to_Policy_Optimization_code/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="http://yunist.cn" target="_blank">云玩家</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/">机器学习</a><a class="post-meta__tags" href="/tags/Python/">Python</a><a class="post-meta__tags" href="/tags/%E5%BC%BA%E5%8C%96%E5%AD%A6%E4%B9%A0/">强化学习</a><a class="post-meta__tags" href="/tags/%E4%BB%A3%E7%A0%81%E5%AE%9E%E4%BE%8B/">代码实例</a><a class="post-meta__tags" href="/tags/Pytorch/">Pytorch</a></div><div class="post_share"><div class="social-share" data-image="/ML/RL/primer/Intro_to_Policy_Optimization_code/Intro_to_Policy_Optimization_code.png" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><!-- - var pagination_cover = prev.cover === false ? prev.randomcover : prev.cover--><a href="/ML/RL/spinningup/begin/"><img class="prev-cover" src="https://cdn.jsdelivr.net/gh/cnyist/blog/ML/RL/spinningup/begin/begin.png" onerror="onerror=null;src='/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">Spinning Up 研究笔记 (零) - 开端</div></div></a></div><!-- - var pagination_cover = next.cover == false ? next.randomcover : next.cover--><div class="next-post pull-right"><a href="/python/another/python_error/"><img class="next-cover" src="https://cdn.jsdelivr.net/gh/cnyist/blog/python/another/python_error/python_error.jpg" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">Python 各种报错解决</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span> 相关推荐</span></div><div class="relatedPosts-list"><div><a href="/ML/RL/spinningup/vpg/" title="Spinning Up 研究笔记 (一) - VPG "><img class="cover" src="https://cdn.jsdelivr.net/gh/cnyist/blog/ML/RL/spinningup/vpg/vpg.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-07-07</div><div class="title">Spinning Up 研究笔记 (一) - VPG </div></div></a></div><div><a href="/ML/RL/primer/RL_demo/" title="强化学习入门 Demo"><img class="cover" src="https://cdn.jsdelivr.net/gh/cnyist/blog/ML/RL/primer/RL_demo/RL_demo.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-04-27</div><div class="title">强化学习入门 Demo</div></div></a></div><div><a href="/ML/RL/spinningup/gym_spaces/" title="Gym 中的 gym.spaces"><img class="cover" src="https://cdn.jsdelivr.net/gh/cnyist/blog/ML/RL/spinningup/gym_spaces/gym_spaces.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-08-09</div><div class="title">Gym 中的 gym.spaces</div></div></a></div><div><a href="/ML/watermelon_book/code/6/" title="《机器学习》 西瓜书实例 第 6 章: 支持向量机"><img class="cover" src="https://cdn.jsdelivr.net/gh/cnyist/blog/ML/watermelon_book/code/6/6.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-02-26</div><div class="title">《机器学习》 西瓜书实例 第 6 章: 支持向量机</div></div></a></div><div><a href="/ML/watermelon_book/code/4/" title="《机器学习》 西瓜书实例 第 4 章: 决策树"><img class="cover" src="https://cdn.jsdelivr.net/gh/cnyist/blog/ML/watermelon_book/code/4/4.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-02-26</div><div class="title">《机器学习》 西瓜书实例 第 4 章: 决策树</div></div></a></div><div><a href="/ML/watermelon_book/code/5/" title="《机器学习》 西瓜书实例 第 5 章: 神经网络"><img class="cover" src="https://cdn.jsdelivr.net/gh/cnyist/blog/ML/watermelon_book/code/5/5.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-02-26</div><div class="title">《机器学习》 西瓜书实例 第 5 章: 神经网络</div></div></a></div></div></div><hr/><div id="post-comment"><div class="comment-head"><div class="comment-headline"><i class="fas fa-comments fa-fw"></i><span> 评论</span></div></div><div class="comment-wrap"><div><div id="twikoo-wrap"></div></div></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="card-info-avatar is-center"><img class="avatar-img" src="https://cdn.jsdelivr.net/gh/cnyist/blog/img/avatar.svg" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/><div class="author-info__name">云玩家</div><div class="author-info__description">云玩家's blog</div></div><div class="card-info-data"><div class="card-info-data-item is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">97</div></a></div><div class="card-info-data-item is-center"><a href="/tags/"><div class="headline">标签</div><div class="length-num">40</div></a></div><div class="card-info-data-item is-center"><a href="/categories/"><div class="headline">分类</div><div class="length-num">27</div></a></div></div><a class="button--animated" id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/xxxxxx"><i class="fab fa-github"></i><span>Follow Me</span></a><div class="card-info-social-icons is-center"><a class="social-icon" href="https://github.com/cnyist" target="_blank" title="Github"><i class="fab fa-github"></i></a><a class="social-icon" href="https://blog.csdn.net/chnyist" target="_blank" title="CSDN"><i class="fas iconfont icon-CN_csdnnet"></i></a><a class="social-icon" href="https://www.zhihu.com/people/cnyist" target="_blank" title="知乎"><i class="fas iconfont icon-zhihu"></i></a><a class="social-icon" href="mailto:yunist@qq.com" target="_blank" title="Email"><i class="fas fa-envelope"></i></a></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn card-announcement-animation"></i><span>公告</span></div><div class="announcement_content">This is my Blog</div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#mlp"><span class="toc-number">1.</span> <span class="toc-text">mlp</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%8F%82%E6%95%B0"><span class="toc-number">1.1.</span> <span class="toc-text">参数</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%A7%A3%E6%9E%90"><span class="toc-number">1.2.</span> <span class="toc-text">解析</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#get-policy"><span class="toc-number">2.</span> <span class="toc-text">get_policy</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%8F%82%E6%95%B0-1"><span class="toc-number">2.1.</span> <span class="toc-text">参数</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%A7%A3%E6%9E%90-1"><span class="toc-number">2.2.</span> <span class="toc-text">解析</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#get-action"><span class="toc-number">3.</span> <span class="toc-text">get_action</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%8F%82%E6%95%B0-2"><span class="toc-number">3.1.</span> <span class="toc-text">参数</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%A7%A3%E6%9E%90-2"><span class="toc-number">3.2.</span> <span class="toc-text">解析</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#compute-loss"><span class="toc-number">4.</span> <span class="toc-text">compute_loss</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%8F%82%E6%95%B0-3"><span class="toc-number">4.1.</span> <span class="toc-text">参数</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%A7%A3%E6%9E%90-3"><span class="toc-number">4.2.</span> <span class="toc-text">解析</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#train-one-epoch"><span class="toc-number">5.</span> <span class="toc-text">train_one_epoch</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E8%A7%A3%E6%9E%90-4"><span class="toc-number">5.1.</span> <span class="toc-text">解析</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#train"><span class="toc-number">6.</span> <span class="toc-text">train</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%8F%82%E6%95%B0-4"><span class="toc-number">6.1.</span> <span class="toc-text">参数</span></a></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><!-- - let post_cover = article.cover--><div class="aside-list-item"><a class="thumbnail" href="/math/set_theory/unique_ordinal/" title="序数的唯一性"><img src="https://cdn.jsdelivr.net/gh/cnyist/blog/math/set_theory/unique_ordinal/unique_ordinal.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="序数的唯一性"/></a><div class="content"><a class="title" href="/math/set_theory/unique_ordinal/" title="序数的唯一性">序数的唯一性</a><time datetime="2022-01-12T08:26:12.000Z" title="发表于 2022-01-12 16:26:12">2022-01-12</time></div></div><!-- - let post_cover = article.cover--><div class="aside-list-item"><a class="thumbnail" href="/math/mathematical_analysis/mathematical_analysis_practice/3/3-1/" title="(史济怀) 数学分析教程上册第 3 版-练习题 3.1"><img src="https://cdn.jsdelivr.net/gh/cnyist/blog/math/mathematical_analysis/mathematical_analysis_practice/3/3-1/math.jpeg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="(史济怀) 数学分析教程上册第 3 版-练习题 3.1"/></a><div class="content"><a class="title" href="/math/mathematical_analysis/mathematical_analysis_practice/3/3-1/" title="(史济怀) 数学分析教程上册第 3 版-练习题 3.1">(史济怀) 数学分析教程上册第 3 版-练习题 3.1</a><time datetime="2021-10-19T00:30:55.000Z" title="发表于 2021-10-19 08:30:55">2021-10-19</time></div></div><!-- - let post_cover = article.cover--><div class="aside-list-item"><a class="thumbnail" href="/math/mathematical_analysis/mathematical_analysis_practice/2/2-11/" title="(史济怀) 数学分析教程上册第 3 版-练习题 2.11"><img src="https://cdn.jsdelivr.net/gh/cnyist/blog/math/mathematical_analysis/mathematical_analysis_practice/2/2-11/2-11.jpeg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="(史济怀) 数学分析教程上册第 3 版-练习题 2.11"/></a><div class="content"><a class="title" href="/math/mathematical_analysis/mathematical_analysis_practice/2/2-11/" title="(史济怀) 数学分析教程上册第 3 版-练习题 2.11">(史济怀) 数学分析教程上册第 3 版-练习题 2.11</a><time datetime="2021-09-25T03:37:28.000Z" title="发表于 2021-09-25 11:37:28">2021-09-25</time></div></div><!-- - let post_cover = article.cover--><div class="aside-list-item"><a class="thumbnail" href="/math/mathematical_analysis/mathematical_analysis_practice/2/2-10/" title="(史济怀) 数学分析教程上册第 3 版-练习题 2.10"><img src="https://cdn.jsdelivr.net/gh/cnyist/blog/math/mathematical_analysis/mathematical_analysis_practice/2/2-10/2-10.jpeg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="(史济怀) 数学分析教程上册第 3 版-练习题 2.10"/></a><div class="content"><a class="title" href="/math/mathematical_analysis/mathematical_analysis_practice/2/2-10/" title="(史济怀) 数学分析教程上册第 3 版-练习题 2.10">(史济怀) 数学分析教程上册第 3 版-练习题 2.10</a><time datetime="2021-09-22T07:36:49.000Z" title="发表于 2021-09-22 15:36:49">2021-09-22</time></div></div><!-- - let post_cover = article.cover--><div class="aside-list-item"><a class="thumbnail" href="/math/mathematical_analysis/mathematical_analysis_practice/2/2-9/" title="(史济怀) 数学分析教程上册第 3 版-练习题 2.9"><img src="https://cdn.jsdelivr.net/gh/cnyist/blog/math/mathematical_analysis/mathematical_analysis_practice/2/2-9/2-9.jpeg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="(史济怀) 数学分析教程上册第 3 版-练习题 2.9"/></a><div class="content"><a class="title" href="/math/mathematical_analysis/mathematical_analysis_practice/2/2-9/" title="(史济怀) 数学分析教程上册第 3 版-练习题 2.9">(史济怀) 数学分析教程上册第 3 版-练习题 2.9</a><time datetime="2021-09-20T08:19:06.000Z" title="发表于 2021-09-20 16:19:06">2021-09-20</time></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2022 By 云玩家</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer><script>document.getElementById('web_bg').style = 'background-image: url("' + get_rand_jsd_pic('cnyist', 'banner@master', '/', 70, '-min.jpg') + '")'</script></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><a id="to_comment" href="#post-comment" title="直达评论"><i class="fas fa-comments"></i></a><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><div class="search-dialog__title" id="local-search-title">本地搜索</div><div id="local-input-panel"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div></div><hr/><div id="local-search-results"></div><span class="search-close-button"><i class="fas fa-times"></i></span></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/js/search/local-search.js"></script><div class="js-pjax"><script>if (!window.MathJax) {
  window.MathJax = {
    loader: {
      source: {
        '[tex]/amsCd': '[tex]/amscd'
      }
    },
    tex: {
      inlineMath: [ ['$','$'], ["\\(","\\)"]],
      tags: 'ams'
    },
    options: {
      renderActions: {
        findScript: [10, doc => {
          for (const node of document.querySelectorAll('script[type^="math/tex"]')) {
            const display = !!node.type.match(/; *mode=display/)
            const math = new doc.options.MathItem(node.textContent, doc.inputJax[0], display)
            const text = document.createTextNode('')
            node.parentNode.replaceChild(text, node)
            math.start = {node: text, delim: '', n: 0}
            math.end = {node: text, delim: '', n: 0}
            doc.math.push(math)
          }
        }, ''],
        addClass: [200,() => {
          document.querySelectorAll('mjx-container:not([display=\'true\']').forEach( node => {
            const target = node.parentNode
            if (!target.classList.contains('has-jax')) {
              target.classList.add('mathjax-overflow')
            }
          })
        }, '', false]
      }
    }
  }
  
  const script = document.createElement('script')
  script.src = 'https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js'
  script.id = 'MathJax-script'
  script.async = true
  document.head.appendChild(script)
} else {
  MathJax.startup.document.state(0)
  MathJax.texReset()
  MathJax.typeset()
}</script><script>(()=>{
  const $countDom = document.getElementById('twikoo-count')
  const init = () => {
    let initData = {
      el: '#twikoo-wrap',
      envId: 'yunist-147dfe',
      region: ''
    }

    if (false) {
      const otherData = false
      initData = Object.assign(initData, otherData)
    }
    
    twikoo.init(initData)
  }

  const getCount = () => {
    twikoo.getCommentsCount({
      envId: 'yunist-147dfe',
      region: '',
      urls: [window.location.pathname],
      includeReply: false
    }).then(function (res) {
      $countDom.innerText = res[0].count
    }).catch(function (err) {
      console.error(err);
    });
  }

  const loadTwikoo = (bool = false) => {
    if (typeof twikoo === 'object') {
      init()
      bool && $countDom && setTimeout(getCount,0)
    } else {
      getScript('https://cdn.jsdelivr.net/npm/twikoo/dist/twikoo.all.min.js').then(()=> {
        init()
        bool && $countDom && setTimeout(getCount,0)
      })
    }
  }

  if ('Twikoo' === 'Twikoo' || !false) {
    if (false) btf.loadComment(document.getElementById('twikoo-wrap'), loadTwikoo)
    else loadTwikoo(true)
  } else {
    window.loadOtherComment = () => {
      loadTwikoo()
    }
  }
})()</script></div><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>